 #!/usr/bin/env python
#encoding:utf-8
#Create：     2013-7-22
#Author：     longlimin


import os
import time
import utils
import traceback
import re
import ins
import hiveEx
import config

class LogParser:
    
    SEP_FIELD = "\t"
    SEP_KEY = "|"
    
    HIVE_FIELD = config.HIVE_SEP_FIELD
    HIVE_ITEMS = config.HIVE_SEP_ITEMS
    HIVE_MAP_KEYS = config.HIVE_SEP_KEYS
    
    START_DATE  = config.LOAD_LOG_START_DATE
    DATE_FORMAT = "%Y-%m-%d"
    IP_REG="^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$"
    
    def __init__(self, code, source_path, des_path):
        '''data_points:数据埋点    source_path:原始日志目录    des_path:解析后的日志目录  '''
        self.code = code
        self.source_path = source_path
        self.des_path = des_path
        self._init()
        self.hiveExe = hiveEx.HiveExcuter(*config.HIVE)
        
    def _init(self):
        self.fields = utils.get_fields(self.code)
        self.CODE_KEYS = []
        print self.fields
        for row in self.fields:
            self.CODE_KEYS.append(row["code"])
        
    def start(self):
        g_start = time.mktime(time.strptime(self.START_DATE, self.DATE_FORMAT))
        path = os.path.join(self.source_path, self.code)
        for item in os.listdir(path):
            tm = None
            try:
                tm = time.strptime(item, self.DATE_FORMAT)
            except:
                ins.errlog("PASER LOG DATE FORMAT ERROR,SOURCE:%s, DATE:%s\n" % (self.source_path,item, traceback.format_exc()))
                continue
            if time.mktime(tm) >= g_start:
                self.dir_ip(path, item)
                
    def dir_ip(self, path, date):
        compile = re.compile(self.IP_REG)
        dtpath = os.path.join(path, date)
        for ip in os.listdir(dtpath):
            match = compile.match(ip)
            if not match:
                ins.errlog("PASER LOG IP ERROR:%s , IP:%s" % (dtpath, ip))
            ippath = os.path.join(dtpath, ip)
            for logname in os.listdir(ippath):
                try:
                    self.run(date, ip, logname)
                except:
                    ins.errlog(traceback.format_exc())
        if os.path.exists(os.path.join(self.des_path, self.code , "%s.txt" % date)):
            self.hiveExe.create_table(tabl_name=self.code, fields=self.fields)
            self.hiveExe.load_data(self.des_path, self.code, date)
    
    def run(self, date, ip, logname):
        if not utils.is_handle(self.code, date, ip, logname):
            logpath = os.path.join(self.source_path, self.code, date, ip, logname)
            wdir   = os.path.join(self.des_path, self.code)
            if not os.path.exists(wdir):
                os.makedirs(wdir)
            wpath  = os.path.join(wdir, "%s.txt" % date)
            
            ins.acclog("PASE BEGIN: %s" % logpath)
            t1 = time.clock()
            flag_lines = self.parse_file(logpath, wpath)
            t2 = time.clock()
            ins.acclog("PASE END: %s" % logpath)
            utils.add_handle(self.code, date, ip, logname, flag_lines, t2-t1, 0)
    
    def parse_file(self, logpath, wpath):
        line_flag = 0
        flog = open(logpath, 'rb')
        lines = flog.readlines()
        des = open(wpath, "a")
        for line in lines:
            line_flag += 1
            des.write("%s\n" % self.parse_line(line))
        flog.close()
        des.close()
        return line_flag
    
    def parse_line(self, line):
        lis = line.replace("\r","").replace("\n","").split(self.SEP_FIELD)
        ldic = {}
        res = ""
        for item in lis:
            ilis = item.split(self.SEP_KEY)
            if len(ilis)==2:
                ldic[ilis[0]]=ilis[1]
            else:
                ldic[ilis[0]]=item[len(ilis[0]):]
        for k in self.CODE_KEYS:
            if k in ldic:
                res += "%s%s" % (ldic.pop(k), self.HIVE_FIELD)
            else:
                res += "null%s" % self.HIVE_FIELD
                
        for k, v in ldic.items():
            res+="%s%s%s%s" % (k, self.HIVE_ITEMS, v, self.HIVE_MAP_KEYS)
        return res.rstrip(self.HIVE_MAP_KEYS)    
            
if __name__ == "__main__":
    logp = LogParser("duba_shouji_maindata", config.SOURCE, config.DES)
    #logp.parse_file("e:\\tmp\\duba_shouji_main.txt")
    print logp.start()
